import re
from datetime import datetime
import time
import requests
from bs4 import BeautifulSoup


def cultivator(message):  # keep many messages in RAM isn't great idea, because this function exists. You can cultivate messages data without any risks right here.
	if 'text' in  message.__dict__:
		print('_'*16, '\n')
		print('Message id:', message.metadata.get('id'))
		print('Message date:', message.metadata.get('date'), '\n')
		print('Message text:', message.text)
		print('_'*16, '\n')


class Scraper():
	def __init__(self, channel, question=None, start=1, stop=None, infoOnly=False, maskCheck=True, availabilityCheck=True):
		if type(channel) != type(str()):
			raise TypeError('"channel" argument must be string.')

		channel = channel.split('/')[-1].split('?')[0].lstrip('@')  # username extractng from link or @username

		if maskCheck and len(set(channel).difference(set('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_'))) != 0:
			raise ValueError('"channel" argument contain illegal character or value does not correct.')

		if availabilityCheck:  # check availability "Preview channel" page
			try:
				preview_page = requests.get('https://t.me/'+channel).content

			except:
				raise ConnectionError('Check your internet connection or firewall settings and try again.')

			preview_page = BeautifulSoup(preview_page, 'lxml')

			if preview_page.find('div', attrs={'class': 'tgme_page_action'}):
				if not preview_page.find('div', attrs={'class': 'tgme_page_context_link_wrap'}):
					raise LookupError('"Channel preview" is not available.')

			else:
				raise ConnectionError('Check your redirecting settings and try again.')

		self.scrape(channel, question, start, stop, infoOnly)


	def scrape(self, username, question=None, start=1, stop=None, infoOnly=False):
		def text_formatting(message_text):  # clean html element and format it to string
			formatted_text = ''
			for html_element in message_text.contents:
				formatted_element = html_element
				if str(type(html_element)) == "<class 'bs4.element.Tag'>":
					formatted_element = BeautifulSoup().new_tag(html_element.name)

					if html_element.name == 'a':
						formatted_element['href'] = html_element['href']

					if len(html_element.contents) > 1:
						formatted_element.string = text_formatting(html_element)

					elif 'class' in html_element.attrs and 'emoji' in html_element.attrs['class']:
						formatted_element = html_element.get_text()

				formatted_text += str(formatted_element)

			return formatted_text.replace('&lt;', '<').replace('&gt;', '>')


		class Channel():  # contain all channel data from "Preview channel" page
			def __init__(self, page_source):
				channel_info_wrap = page_source.find('section', attrs={'class': 'tgme_right_column'})

				c_counter_wraps = channel_info_wrap.find('div', attrs={'class': 'tgme_channel_info_counters'})
				c_counter_wraps = c_counter_wraps.find_all('div', attrs={'class': 'tgme_channel_info_counter'})

				self.counters = {}

				for c_counter_wrap in c_counter_wraps:
					c_type = c_counter_wrap.find('span', attrs={'class': 'counter_type'}).get_text()
					c_value = c_counter_wrap.find('span', attrs={'class': 'counter_value'}).get_text()
					self.counters.update({c_type: c_value})

				c_description = channel_info_wrap.find('div', attrs={'class': 'tgme_channel_info_description'})
				self.description = text_formatting(c_description) if c_description else None

				channel_info_wrap = channel_info_wrap.find('div', attrs={'class': 'tgme_channel_info_header'})

				c_image_wrap = channel_info_wrap.find('img')
				self.image = c_image_wrap.get('src') if c_image_wrap else None

				self.title = channel_info_wrap.find('div', attrs={'class': 'tgme_channel_info_header_title'}).get_text()

				# if you found public channels with labels "fake", "scam", etc, please send me their links
				c_label_wraps = channel_info_wrap.find('div', attrs={'class': 'tgme_channel_info_header_labels'}).find_all('i')

				self.labels = {}

				for label_wrap in c_label_wraps:
					self.labels.update({'verified': True if 'verified-icon' in label_wrap['class'] else False})
					self.labels.update({'fake': True if 'fake-icon' in label_wrap['class'] else False})  # I didn't test this
					self.labels.update({'scam': True if 'scam-icon' in label_wrap['class'] else False})  # I didn't test this

				self.username = channel_info_wrap.find('div', attrs={'class': 'tgme_channel_info_header_username'}).get_text().lstrip('@')


		url = 'https://t.me/s/' + username
		separator = '?'

		# this is a bad way. I going to rewrite question code block
		if question:  # I don't recommend to use telegram search. Better way is creating your own search engine or use regular expressions manually
			chars_table = {'@': '40', '#': '23', '$': '24', '': '25', '&': '26', '+': '2B', '=': '3D', ',': '2C', '/': '2F', '?': '3F', ';': '3B', ':': '3A'}
			encoded_question = ''

			for c in question:  # find all tabled chars in question and replace it
				if c in chars_table.values():
					encoded_question += '%'+chars_table.get(c)

				else:
						encoded_question += c

			question = True
			url = url + '?q=' + encoded_question
			separator = '&'

		big_value = 1_000_000

		if not stop:
			extended_url = url
			stop = big_value

		else:
			extended_url = url + separator + 'before=' + str(stop+1)


		class Message():  # contain all message data
			def __init__(self, message_wrap):
				self.get_source(message_wrap)
				self.get_message_text(message_wrap)
				self.get_media(message_wrap)
				self.get_file(message_wrap)
				self.get_poll(message_wrap)
				self.get_link_preview(message_wrap)
				self.get_keyboard(message_wrap)
				self.get_metadata(message_wrap)


			def get_source(self, message):  # get forwarding info
				try:
					f_data = message.find('a', attrs={'class': 'tgme_widget_message_forwarded_from_name'})
					f_title = f_data.get_text()

					try:
						f_link = f_data.get('href')

					except:
						f_link = None

					self.source = {'title': f_title, 'url': f_link}

				except:
					pass


			def get_message_text(self, message):
				try:
					message_wrap_text = message.find('div', attrs={'class': 'tgme_widget_message_text'})
					self.text = text_formatting(message_wrap_text)

				except:
					pass

			def get_image(self, message, img_type=0):  # -1 - channel 0 - standart, 1 - group,  2 - preview, 3 - site-preview
				if img_type in (2, 3):
					tag_class = 'link_preview' + ('_right' if img_type == 3 else '') + '_image'
					html_tag = 'i'

				else:
					html_tag = 'a'
					tag_class = 'tgme_widget_message_' + ('photo_wrap' if img_type in (0, 1) else 'service_photo')

				try:
					img_wrap = message.find(html_tag, attrs={'class': tag_class}) if img_type != 1 else message

					if img_type != -1:
						img_style = img_wrap.get('style')
						img_link = re.search(r'background-image:url\(([\'"])(.+)\1\)', img_style)[2]

					else:
						img_link = img_wrap.find('img').get('src')

					if img_type in (-1, 0):
						self.media = ({'type': 'image', 'url': img_link})

					else:
						return {'type': 'image', 'url': img_link}

				except:
					if img_type not in (-1, 0):
						return None


			def get_video(self, message, vid_type=0):  # 0 - standart, 1 - group, 2 - preview, 3 - round
				tag_class_prefix = 'tgme_widget_message_'
				time_class_prefix = 'message_'

				if vid_type in (0, 1):
					html_tag = 'a'

				else:
					html_tag = 'div'

					if vid_type == 2:
						tag_class_prefix = 'link_preview_'

					else:
						tag_class_prefix += 'round'
						time_class_prefix = 'tgme_widget_' + time_class_prefix + 'round'

				try:
					vid_wrap = message.find(html_tag, attrs={'class': f'{tag_class_prefix}video_player'}) if vid_type != 1 else message

					vid_thumb = vid_wrap.find('i', attrs={'class': f'{tag_class_prefix}video_thumb'}).get('style')
					vid_thumb = re.search(r'background-image:url\(([\'"])(.+)\1\)', vid_thumb)[2]

					vid_source = vid_wrap.find('video', attrs={'class': f'{tag_class_prefix}video'})
					vid_source = vid_source.get('src') if vid_source else None

					vid_duration = vid_wrap.find('time', attrs={'class': f'{time_class_prefix}video_duration'})
					vid_duration = vid_duration.get_text() if vid_duration else None

					if vid_type in (0, 3):
						self.media = ({'type': 'video', 'thumbnail': vid_thumb, 'url': vid_source, 'duration': vid_duration})

					else:
						return {'type': 'video', 'thumbnail': vid_thumb, 'url': vid_source, 'duration': vid_duration}

				except:
					if vid_type not in (0, 3):
						return None


			def get_voice(self, message):
				try:
					voice_data = message.find('a', attrs={'class': 'tgme_widget_message_voice_player'})

					v_audio = voice_data.find('audio', attrs={'class': 'tgme_widget_message_voice'}).get('src')
					v_duration = voice_data.find('time', attrs={'class': 'tgme_widget_message_voice_duration'}).get_text()

					self.media = ({'type': 'audio', 'url': v_audio, 'duration': v_duration})

				except:
					pass


			def get_media(self, message):  # get single images, videos or contained in groupmedia wrap, roundvideo and voice message info
				try:
					media_wraps = message.find('div', attrs={'class': 'tgme_widget_message_grouped_layer'}).contents

					media = []

					for media_wrap in media_wraps:
						if str(type(media_wrap)) == "<class 'bs4.element.Tag'>":
							media.append(self.get_image(media_wrap, img_type=1) or self.get_video(media_wrap, vid_type=1))

					self.media = tuple(media)

				except:
					self.get_image(message) or self.get_video(message) or self.get_image(message, img_type=-1) or self.get_video(message, vid_type=3) or self.get_voice(message)


			def get_file(self, message):
				try:
					doc_data = message.find('div', attrs={'class': 'tgme_widget_message_document'})

					d_title = doc_data.find('div', attrs={'class': 'tgme_widget_message_document_title'}).get_text()
					d_extra = doc_data.find('div', attrs={'class': 'tgme_widget_message_document_extra'}).get_text()

					self.file = {'title': d_title, 'extra': d_extra}

				except:
					pass


			def get_poll(self, message):
				try:
					poll_data = message.find('div', attrs={'class': 'tgme_widget_message_poll'})

					p_title = poll_data.find('div', attrs={'class': 'tgme_widget_message_poll_question'}).get_text()
					p_type = poll_data.find('div', attrs={'class': 'tgme_widget_message_poll_type'}).get_text()
					p_options = poll_data.find('a', attrs={'class': 'tgme_widget_message_poll_options'})
					p_options = p_options.find_all('div', attrs={'class': 'tgme_widget_message_poll_option'})
					p_opts = []

					for p_option in p_options:
						opt_percent = int(p_option.find('div', attrs={'class': 'tgme_widget_message_poll_option_percent'}).get_text().rstrip('%'))
						opt_text = p_option.find('div', attrs={'class': 'tgme_widget_message_poll_option_text'}).get_text()
						p_opts.append({'percent': opt_percent, 'text': opt_text})

					self.poll = {'title': p_title, 'type': p_type, 'answers': tuple(p_opts)}

				except:
					pass


			def get_link_preview(self, message):
				try:
					preview_data = message.find('a', attrs={'class': 'tgme_widget_message_link_preview'})
					s_link = preview_data.get('href')

					try:
						s_thumb = self.get_image(preview_data, img_type=3).get('url')
						s_name = preview_data.find('div', attrs={'class': 'link_preview_site_name'}).get_text()
						s_title = preview_data.find('div', attrs={'class': 'link_preview_title'}).get_text()
						s_description = preview_data.find('div', attrs={'class': 'link_preview_description'}).get_text()

						self.link_preview = {'url': s_link, 'site-preview': {'name': s_name, 'thumbnail': s_thumb, 'title': s_title, 'description': s_description}}

					except:
						self.link_preview = {'url': s_link, 'media': self.get_image(preview_data, img_type=2) or self.get_video(preview_data, vid_type=2)}

				except:
					pass


			def get_metadata(self, message):
				footer_data = message.find('div', attrs={'class': 'tgme_widget_message_footer'})

				f_meta = footer_data.find('span', attrs={'class': 'tgme_widget_message_meta'})
				f_info = f_meta.find('a', attrs={'class': 'tgme_widget_message_date'})

				f_id = int(f_info.get('href').rsplit('/', 1)[-1])

				f_date = f_info.find('time', attrs={'class': 'time'}).get('datetime')
				f_date = datetime.strptime(f_date, '%Y-%m-%dT%H:%M:%S%z')

				f_edited = True if 'edited' in f_meta.contents[0] else False

				f_views = footer_data.find('span', attrs={'class': 'tgme_widget_message_views'})
				f_views = f_views.get_text() if f_views else None

				f_voters = footer_data.find('span', attrs={'class': 'tgme_widget_message_voters'})
				f_voters = f_voters.get_text() if f_voters else None
				if 'poll' in self.__dict__:
					self.poll.update({'voters': f_voters})

				self.metadata = {'id': f_id, 'date': f_date, 'views': f_views, 'edited': f_edited}


			def get_keyboard(self, message):
				try:
					keyboard_data = message.find('div', attrs={'class': 'tgme_widget_message_inline_keyboard'})
					k_rows_data = keyboard_data.find_all('div', attrs={'class': 'tgme_widget_message_inline_row'})
					k_rows = []

					for k_row_data in k_rows_data:
						k_row = {}
						buttons_data = k_row_data.find_all('a', attrs={'class': 'tgme_widget_message_inline_button'})

						for button_data in buttons_data:
							b_text = button_data.find('span', attrs={'class': 'tgme_widget_message_inline_button_text'}).get_text()
							b_href = button_data.get('href')
							k_row.update({'text': b_text, 'url': b_href})

						k_rows.append(k_row)

					self.keyboard = tuple(k_rows)

				except:
					pass


		informated = False
		while True:
			try:
				page_source = requests.get(extended_url).content

			except:
				time.sleep(5)
				page_source = requests.get(extended_url).content

			channel_page = BeautifulSoup(page_source, 'lxml')
			if not informated:
				self.channel = Channel(channel_page)
				informated = True
				if infoOnly:
					break

			history_wrap = channel_page.find('section', attrs={'class': 'tgme_channel_history'})
			message_wraps = history_wrap.find_all('div', attrs={'class': 'tgme_widget_message'})[::-1]

			# <a>   tme_messages_more js-messages_more
			# href="/s/channel?before={number}"
			# data-before="{number}"

			min = big_value

			for message_wrap in message_wraps:
				message = Message(message_wrap)
				min = message.metadata.get('id') if min > message.metadata.get('id') else min

				if message.metadata.get('id') in range(start, stop+1):
					try:
						cultivator(message)

					except Exception as ex:
						print(ex)
						print('Congradulations! Your function succesfully failed!')
						return 1

			if min in range(start+1, stop):
				if question and len(message_wraps) < 20:  # this thing will be removed asap
					break

				extended_url = url + f'{separator}before={str(min)}'
				# time.sleep(5)

			else:
				break


if __name__ == '__main__':
	channel = input('Enter a channel username/link: ')
	question = input('Enter a question: ')
	Scraper(channel, question)

